#!/bin/bash

#获取jobset 且创建时间，如果超过12小时，则清理

gc_hour=12

colorEcho() {
    local color=$1
    shift
    echo -e "\e[${color}m$*\e[0m"
}

for jobset in `kubectl get jobset -n llm-finetune | grep ^chat |  awk -F " " '{print $1}'`
do
#    echo ${jobset}
    timestamp=`kubectl describe  jobset -n llm-finetune ${jobset}  | grep "Creation Timestamp"`
#    echo ${timestamp}
    time_only=${timestamp##*: }
#   echo ${time_only}
    timestamp_by_sec=$(date -u -d "$time_only" +%s)
    current_timestamp=$(date -u +%s)
    time_run_seconds=$((current_timestamp - timestamp_by_sec))
#    echo "${time_run_seconds} seconds"
    time_run_hours=$(($time_run_seconds/ 3600))
    if [ $time_run_hours -gt $gc_hour ]; then
    	colorEcho "31" "The $jobset runtime is $time_run_hours hours should be clean."
	kubectl delete jobset -n llm-finetune ${jobset}
    else
    	colorEcho "32" "The $jobset runtime is $time_run_hours hours."
    fi
done

